package com.simple.database.pageprocess;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHost;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.scheduler.component.HashSetDuplicateRemover;

import com.alibaba.fastjson.JSONObject;
import com.simple.database.pojo.Blog;
import com.simple.database.utils.RedisUtil;

public class BlogTianyaPageProcess implements PageProcessor{

	private Site site = Site.me().setHttpProxy(new HttpHost("202.109.207.103",8888)).setRetryTimes(3).setSleepTime(5000).setCharset("utf-8").
			setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36");
	
	Logger logger =LoggerFactory.getLogger(this.getClass());
	
	// 计数器
	private final AtomicLong count = new AtomicLong(0);

	//加载spring配置文件
	ApplicationContext ac = new ClassPathXmlApplicationContext("spring/applicationContext*.xml");

	 @Override
	    public void process(Page page) {
		 	Blog blog = new Blog();
	        page.addTargetRequests(page.getHtml().links().regex("http://blog\\.tianya\\.cn/post-\\d+-\\d+-\\d+\\.shtml").all());
	        page.putField("title", page.getHtml().xpath("//div[@class='article']/h2/a/text()").toString());
	        page.putField("url", page.getUrl());
	        String url = String.valueOf(page.getResultItems().get("url")).trim();
	        String JsonValue = RedisUtil.get(url);
	        if (page.getResultItems().get("title")==null){
	            //标题为nulll则跳过
	            page.setSkip(true);
	            logger.warn("===>(标题为空)跳过url:"+url);
	        }else if(StringUtils.isNoneBlank(JsonValue)){
	        	//对应的url有数据则跳过
	        	 page.setSkip(true);
	        	 logger.warn("===>(redis中已存在)跳过url:"+url);
	        }else{
	        	count.incrementAndGet();
	        	System.out.println("抓取到的第："+count.get()+"条有效数据");
	        	//解析html获取数据
		        page.putField("author", page.getHtml().xpath("//a[@class='userName']/text()").toString());
	        	page.putField("views", page.getHtml().xpath("//em[@id='AllVisitCounter']/text()"));
//	        	page.putField("2222views222", page.getHtml().xpath("//em[@id='PostClick']/text()"));
	        	page.putField("comments", page.getHtml().xpath("//div[@class='article-categories pos-relative']/a/@href='#allcomments'/text()").toString().replace("评论:", ""));
	        	page.putField("writeTime", page.getHtml().xpath("//div[@class='article-tag pos-relative cf']/span/text()").toString().substring(0,16));
				blog.setTitle(String.valueOf(page.getResultItems().get("title")).trim());
				blog.setUrl(String.valueOf(page.getResultItems().get("url")).trim());
				blog.setAuthor(String.valueOf(page.getResultItems().get("author")).trim());
				String views =String.valueOf(page.getResultItems().get("views").toString().trim());
				blog.setViews(Integer.valueOf(views.trim()));
				String comments = String.valueOf(page.getResultItems().get("comments"));
				blog.setComments(Integer.valueOf(comments.trim()));
				Date writeDate = null;
				
				try {
					writeDate = new SimpleDateFormat("yyyy-MM-dd HH:mm").parse(String.valueOf(page.getResultItems().get("writeTime")).trim());
				} catch (Exception e1) {
					e1.printStackTrace();
					logger.error("时间转换出错：{}",e1.getMessage());
				}
				blog.setWriteTime(writeDate);
				blog.setInsertTime(new Date());
//				blog.setSid(2);
				RedisUtil.setex(blog.getUrl(), JSONObject.toJSONString(blog),8640000);//100天
//				RedisUtil.set(blog.getUrl(), JSONObject.toJSONString(blog));
				//this.redisService.set(blog.getUrl(), JSONObject.toJSONString(blog));
//				logger.warn("====>blog info:{}",blog);;
				//以后用nosql做
//				blogMapper.insert(blog);
	        }
	    }

	    @Override
	    public Site getSite() {
	        return site;
	    }

	    public static void main(String[] args) {
	        Spider.create(new BlogTianyaPageProcess())
	        .addUrl("http://blog.tianya.cn/post-4014292-124250253-1.shtml")
	        .setScheduler(new QueueScheduler().setDuplicateRemover(new HashSetDuplicateRemover()))
	        .setScheduler(new FileCacheQueueScheduler("E:/temp/webmagic/tianya"))
	        .thread(1)
	        .run();
	    }

}
